{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "67c1e6b1",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "下面的例子将展示词向量标准工具包——gensim提供的词嵌入，并展示词嵌入如何表示词的相似度。\n",
    "<!-- https://nlp.stanford.edu/projects/glove/ -->"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5c5a740a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pprint\n",
    "\n",
    "from gensim.models import KeyedVectors\n",
    "\n",
    "# 从GloVe官网下载GloVe向量，此处使用的是glove.6B.zip\n",
    "# 解压缩zip文件并将以下路径改为解压后对应文件的路径\n",
    "model = KeyedVectors.load_word2vec_format('./archive'+\\\n",
    "    '/glove.6B.100d.txt', binary=False, no_header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "01a2e4a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[('movie', 0.9055121541023254),\n",
      " ('films', 0.8914433717727661),\n",
      " ('directed', 0.8124362826347351),\n",
      " ('documentary', 0.8075793981552124),\n",
      " ('drama', 0.7929168939590454),\n",
      " ('movies', 0.7889865040779114),\n",
      " ('comedy', 0.7842751145362854),\n",
      " ('starring', 0.7573285102844238),\n",
      " ('cinema', 0.7419455647468567),\n",
      " ('hollywood', 0.7307389974594116)]\n",
      "[('vehicle', 0.8630837798118591),\n",
      " ('truck', 0.8597878813743591),\n",
      " ('cars', 0.837166965007782),\n",
      " ('driver', 0.8185911178588867),\n",
      " ('driving', 0.781263530254364),\n",
      " ('motorcycle', 0.7553156614303589),\n",
      " ('vehicles', 0.7462257146835327),\n",
      " ('parked', 0.74594646692276),\n",
      " ('bus', 0.737270712852478),\n",
      " ('taxi', 0.7155269384384155)]\n"
     ]
    }
   ],
   "source": [
    "# 使用most_similar()找到词表中距离给定词最近（最相似）的n个词\n",
    "pprint.pprint(model.most_similar('film'))\n",
    "pprint.pprint(model.most_similar('car'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8b62f7ad",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "japanese\n",
      "panda\n",
      "longest\n",
      "terrible\n",
      "queen\n"
     ]
    }
   ],
   "source": [
    "# 利用GloVe展示一个类比的例子\n",
    "def analogy(x1, x2, y1):\n",
    "    # 寻找top-N最相似的词。\n",
    "    result = model.most_similar(positive=[y1, x2], negative=[x1])\n",
    "    return result[0][0]\n",
    "\n",
    "print(analogy('china', 'chinese', 'japan'))\n",
    "print(analogy('australia', 'koala', 'china'))\n",
    "print(analogy('tall', 'tallest', 'long'))\n",
    "print(analogy('good', 'fantastic', 'bad'))\n",
    "print(analogy('man', 'woman', 'king'))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c308cee",
   "metadata": {},
   "source": [
    "下面将展示word2vec的代码，包括文本预处理、skipgram算法的实现、以及使用PyTorch进行优化。这里使用《小王子》这本书作为训练语料。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "590fc408",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 安装NLTK，使用如下代码下载punkt组件\n",
    "#import nltk\n",
    "#nltk.download('punkt')\n",
    "\n",
    "from nltk.tokenize import sent_tokenize, word_tokenize\n",
    "from collections import defaultdict\n",
    "\n",
    "# 使用类管理数据对象，包括文本读取、文本预处理等\n",
    "class TheLittlePrinceDataset:\n",
    "    def __init__(self, tokenize=True):\n",
    "        # 利用NLTK函数进行分句和分词\n",
    "        text = open('the little prince.txt', 'r', encoding='utf-8').read()\n",
    "        if tokenize:\n",
    "            self.sentences = sent_tokenize(text.lower())\n",
    "            self.tokens = [word_tokenize(sent) for sent in self.sentences]\n",
    "        else:\n",
    "            self.text = text\n",
    "\n",
    "    def build_vocab(self, min_freq=1):\n",
    "        # 统计词频\n",
    "        frequency = defaultdict(int)\n",
    "        for sentence in self.tokens:\n",
    "            for token in sentence:\n",
    "                frequency[token] += 1\n",
    "        self.frequency = frequency\n",
    "\n",
    "        # 加入<unk>处理未登录词，加入<pad>用于对齐变长输入进而加速\n",
    "        self.token2id = {'<unk>': 1, '<pad>': 0}\n",
    "        self.id2token = {1: '<unk>', 0: '<pad>'}\n",
    "        for token, freq in sorted(frequency.items(), key=lambda x: -x[1]):\n",
    "            # 丢弃低频词\n",
    "            if freq > min_freq:\n",
    "                self.token2id[token] = len(self.token2id)\n",
    "                self.id2token[len(self.id2token)] = token\n",
    "            else:\n",
    "                break\n",
    "\n",
    "    def get_word_distribution(self):\n",
    "        distribution = np.zeros(vocab_size)\n",
    "        for token, freq in self.frequency.items():\n",
    "            if token in dataset.token2id:\n",
    "                distribution[dataset.token2id[token]] = freq\n",
    "            else:\n",
    "                # 不在词表中的词按<unk>计算\n",
    "                distribution[1] += freq\n",
    "        distribution /= distribution.sum()\n",
    "        return distribution\n",
    "\n",
    "    # 将分词结果转化为索引表示\n",
    "    def convert_tokens_to_ids(self, drop_single_word=True):\n",
    "        self.token_ids = []\n",
    "        for sentence in self.tokens:\n",
    "            token_ids = [self.token2id.get(token, 1) for token in sentence]\n",
    "            # 忽略只有一个token的序列，无法计算loss\n",
    "            if len(token_ids) == 1 and drop_single_word:\n",
    "                continue\n",
    "            self.token_ids.append(token_ids)\n",
    "        \n",
    "        return self.token_ids\n",
    "\n",
    "dataset = TheLittlePrinceDataset()\n",
    "dataset.build_vocab(min_freq=1)\n",
    "sentences = dataset.convert_tokens_to_ids()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "efc882de",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(76044, 2) [[  4  16]\n",
      " [  4  19]\n",
      " [ 16   4]\n",
      " ...\n",
      " [130   3]\n",
      " [  3  86]\n",
      " [  3 130]]\n"
     ]
    }
   ],
   "source": [
    "# 遍历所有的中心词-上下文词对\n",
    "window_size = 2\n",
    "data = []\n",
    "\n",
    "for sentence in sentences:\n",
    "    for i in range(len(sentence)):\n",
    "        for j in range(i-window_size, i+window_size+1):\n",
    "            if j == i or j < 0 or j >= len(sentence):\n",
    "                continue\n",
    "            center_word = sentence[i]\n",
    "            context_word = sentence[j]\n",
    "            data.append([center_word, context_word])\n",
    "\n",
    "# 需要提前安装numpy\n",
    "import numpy as np\n",
    "data = np.array(data)\n",
    "print(data.shape, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "30903b3d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# 需要提前安装PyTorch\n",
    "import torch\n",
    "from torch import nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "# 实现skipgram算法，使用对比学习计算损失\n",
    "class SkipGramNCE(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_size, distribution,\\\n",
    "                 neg_samples=20):\n",
    "        super(SkipGramNCE, self).__init__()\n",
    "        print(f'vocab_size = {vocab_size}, embed_size = {embed_size}, '+\\\n",
    "              f'neg_samples = {neg_samples}')\n",
    "        self.input_embeddings = nn.Embedding(vocab_size, embed_size)\n",
    "        self.output_embeddings = nn.Embedding(vocab_size, embed_size)\n",
    "        distribution = np.power(distribution, 0.75)\n",
    "        distribution /= distribution.sum()\n",
    "        self.distribution = torch.tensor(distribution)\n",
    "        self.neg_samples = neg_samples\n",
    "        \n",
    "    def forward(self, input_ids, labels):\n",
    "        i_embed = self.input_embeddings(input_ids)\n",
    "        o_embed = self.output_embeddings(labels)\n",
    "        batch_size = i_embed.size(0)\n",
    "        n_words = torch.multinomial(self.distribution, batch_size * \\\n",
    "            self.neg_samples, replacement=True).view(batch_size, -1)\n",
    "        n_embed = self.output_embeddings(n_words)\n",
    "        pos_term = F.logsigmoid(torch.sum(i_embed * o_embed, dim=1))\n",
    "        # 负采样，用于对比学习\n",
    "        neg_term = F.logsigmoid(- torch.bmm(n_embed, \\\n",
    "            i_embed.unsqueeze(2)).squeeze())\n",
    "        neg_term = torch.sum(neg_term, dim=1)\n",
    "        loss = - torch.mean(pos_term + neg_term)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1d9da6c8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.00000000e+00 5.43983724e-02 5.34295679e-02 ... 9.68804495e-05\n",
      " 9.68804495e-05 9.68804495e-05]\n",
      "vocab_size = 1078, embed_size = 128, neg_samples = 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "epoch-99, loss=3.0434: 100%|█| 100/100 [05:02<00:00,  3.03s/\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjIAAAGwCAYAAACzXI8XAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAABUGklEQVR4nO3dd3yT1f4H8E/SNOlMSksntOwNZY+CDAVZioB4UQQFr4poUYYDuVe91+soPxc4EPVeBQeIogwXIiAUwbZA2atQVgtd0NKkM22S8/ujzdNG2tKRJ2nK5/165fUiT548OTzW5sM533OOQgghQEREROSClM5uABEREVF9McgQERGRy2KQISIiIpfFIENEREQui0GGiIiIXBaDDBEREbksBhkiIiJyWSpnN0BuFosFaWlp8PX1hUKhcHZziIiIqBaEEMjLy0NYWBiUyur7XZp8kElLS0N4eLizm0FERET1kJqaipYtW1b7epMPMr6+vgDKboRWq3Vya4iIiKg2DAYDwsPDpe/x6jT5IGMdTtJqtQwyRERELuZGZSEs9iUiIiKXxSBDRERELotBhoiIiFwWgwwRERG5LAYZIiIiclkMMkREROSyGGSIiIjIZTHIEBERkctqNEFmyZIlUCgUmD9/vnSsuLgY0dHRCAgIgI+PD6ZMmYLMzEznNZKIiIgalUYRZPbt24ePP/4YkZGRNscXLFiAH3/8EevWrUNsbCzS0tJw9913O6mVRERE1Ng4Pcjk5+dj+vTp+O9//4tmzZpJx/V6PT799FO88847uO2229C3b1+sXLkSf/75J+Lj46u9ntFohMFgsHkQERFR0+T0IBMdHY077rgDo0aNsjmemJiI0tJSm+OdO3dGREQE4uLiqr1eTEwMdDqd9ODO10RERE2XU4PM2rVrceDAAcTExFz3WkZGBtRqNfz8/GyOBwcHIyMjo9prLl68GHq9Xnqkpqbau9kAAKPJjIvZBcjON8pyfSIiIroxp+1+nZqainnz5mHr1q3w8PCw23U1Gg00Go3drledZ9YdwY+H0/DP8V3w6LC2sn8eERERXc9pPTKJiYnIyspCnz59oFKpoFKpEBsbi/feew8qlQrBwcEoKSlBbm6uzfsyMzMREhLinEZXEqYrC19p+iInt4SIiOjm5bQemZEjR+Lo0aM2xx566CF07twZixYtQnh4ONzd3bF9+3ZMmTIFAJCUlISUlBRERUU5o8k2QsuDTIa+2MktISIiunk5Lcj4+vqie/fuNse8vb0REBAgHX/44YexcOFC+Pv7Q6vV4sknn0RUVBQGDRrkjCbbCPXzBACkMcgQERE5jdOCTG0sXboUSqUSU6ZMgdFoxJgxY/Dhhx86u1kAKnpk0nM5tEREROQsCiGEcHYj5GQwGKDT6aDX66HVau123St5RvR/bRsUCiDplXFQq5w+k52IiKjJqO33N7996ynAWw21mxJCAJkGDi8RERE5A4NMPSmVCoRYC34ZZIiIiJyCQaYBrEEmjXUyRERETsEg0wDWtWTSOXOJiIjIKRhkGsA6BZszl4iIiJyDQaYB2CNDRETkXAwyDRCiK++RYZAhIiJyCgaZBpAWxeN+S0RERE7BINMAYeU1MlfzS2A0mZ3cGiIiopsPg0wDNPNyh6Z8RV9uHklEROR4DDINoFAopF4Z1skQERE5HoNMA4VoWSdDRETkLAwyDRTqZ13dlz0yREREjsYg00Bh0hRs9sgQERE5GoNMA1l7ZFjsS0RE5HgMMg0UquPQEhERkbMwyDRQKIeWiIiInIZBpoGsNTLXCktRVMJF8YiIiByJQaaBtJ4qeKndAAAZBg4vERERORKDTAMpFAqEWPdcyuXwEhERkSMxyNiBdXgpjTOXiIiIHIpBxg5C2SNDRETkFAwydhDqxx4ZIiIiZ2CQsQNrj0wGp2ATERE5FIOMHUhDS+yRISIicigGGTsIsw4tsUaGiIjIoRhk7MDaI2MoNqHAaHJya4iIiG4eDDJ24OvhDh+NCgCHl4iIiByJQcZOKupkOLxERETkKAwydmKdgp3OXbCJiIgchkHGTsLKe2TS2CNDRETkMAwydlKx3xJ7ZIiIiByFQcZOrPstpXMHbCIiIodhkLGTUD/ut0RERORoTg0yK1asQGRkJLRaLbRaLaKiorB582bp9REjRkChUNg85syZ48QWVy/U2iPD6ddEREQOo3Lmh7ds2RJLlixBhw4dIITA559/jokTJ+LgwYPo1q0bAODRRx/Ff/7zH+k9Xl5ezmpujYK1GgBAvtGEohIzPNVuTm4RERFR0+fUIDNhwgSb56+99hpWrFiB+Ph4Kch4eXkhJCSk1tc0Go0wGo3Sc4PBYJ/G3oCHe0VwKTFZGGSIiIgcoNHUyJjNZqxduxYFBQWIioqSjq9evRrNmzdH9+7dsXjxYhQWFtZ4nZiYGOh0OukRHh4ud9MBACqlQvpzidnikM8kIiK62Tm1RwYAjh49iqioKBQXF8PHxwcbNmxA165dAQD3338/WrVqhbCwMBw5cgSLFi1CUlIS1q9fX+31Fi9ejIULF0rPDQaDQ8KMQqGA2k2JErMFpQwyREREDuH0INOpUyccOnQIer0e3333HWbOnInY2Fh07doVs2fPls7r0aMHQkNDMXLkSJw9exbt2rWr8noajQYajcZRzbfh7qZAiRkMMkRERA7i9KEltVqN9u3bo2/fvoiJiUHPnj3x7rvvVnnuwIEDAQDJycmObGKtuavKbmepWTi5JURERDcHpweZv7JYLDbFupUdOnQIABAaGurAFtWeu5s1yLBHhoiIyBGcOrS0ePFijBs3DhEREcjLy8OaNWuwc+dObNmyBWfPnsWaNWswfvx4BAQE4MiRI1iwYAGGDRuGyMhIZza7WmoGGSIiIodyapDJysrCgw8+iPT0dOh0OkRGRmLLli24/fbbkZqaim3btmHZsmUoKChAeHg4pkyZghdeeMGZTa6Ryq1s5hKDDBERkWM4Nch8+umn1b4WHh6O2NhYB7am4axDSyUm1sgQERE5QqOrkXFlrJEhIiJyLAYZO1JzaImIiMihGGTsiD0yREREjsUgY0cVQYY1MkRERI7AIGNHFQvisUeGiIjIERhk7Ig1MkRERI7FIGNHKmX59GsOLRERETkEg4wdSUNLJvbIEBEROQKDjB25c2iJiIjIoRhk7Ih7LRERETkWg4wdSVsUsEaGiIjIIRhk7IgL4hERETkWg4wduavKamRMDDJEREQOwSBjR2qu7EtERORQDDJ2VFEjwx4ZIiIiR2CQsSOVdfo115EhIiJyCAYZO+L0ayIiIsdikLEj7n5NRETkWAwydsQaGSIiIsdikLEjblFARETkWAwydqRWsUaGiIjIkRhk7Ig1MkRERI7FIGNH3KKAiIjIsRhk7Ig1MkRERI7FIGNHUo+MiUNLREREjsAgY0ccWiIiInIsBhk7sg4tcR0ZIiIix2CQsSP2yBARETkWg4wdVawjwxoZIiIiR2CQsaOKYl/2yBARETkCg4wdSdOvLQwyREREjsAgY0dqruxLRETkUAwydqQqDzJmi4DZwjBDREQkN6cGmRUrViAyMhJarRZarRZRUVHYvHmz9HpxcTGio6MREBAAHx8fTJkyBZmZmU5scc2sQ0sAZy4RERE5glODTMuWLbFkyRIkJiZi//79uO222zBx4kQcP34cALBgwQL8+OOPWLduHWJjY5GWloa7777bmU2ukbXYF2CQISIicgSFEKJRjYH4+/vjzTffxD333IPAwECsWbMG99xzDwDg1KlT6NKlC+Li4jBo0KBaXc9gMECn00Gv10Or1crZdJgtAu3+8QsA4MCLt8PfWy3r5xERETVVtf3+bjQ1MmazGWvXrkVBQQGioqKQmJiI0tJSjBo1Sjqnc+fOiIiIQFxcXLXXMRqNMBgMNg9HcVMq4KbkxpFERESO4vQgc/ToUfj4+ECj0WDOnDnYsGEDunbtioyMDKjVavj5+dmcHxwcjIyMjGqvFxMTA51OJz3Cw8Nl/hvYkrYp4FoyREREsnN6kOnUqRMOHTqEhIQEPP7445g5cyZOnDhR7+stXrwYer1eeqSmptqxtTdmrZMxcdYSERGR7FTOboBarUb79u0BAH379sW+ffvw7rvv4t5770VJSQlyc3NtemUyMzMREhJS7fU0Gg00Go3cza6WmvstEREROYzTe2T+ymKxwGg0om/fvnB3d8f27dul15KSkpCSkoKoqCgntrBm1h4ZDi0RERHJz6k9MosXL8a4ceMQERGBvLw8rFmzBjt37sSWLVug0+nw8MMPY+HChfD394dWq8WTTz6JqKioWs9YcgaVG4t9iYiIHMWpQSYrKwsPPvgg0tPTodPpEBkZiS1btuD2228HACxduhRKpRJTpkyB0WjEmDFj8OGHHzqzyTfEbQqIiIgcx6lB5tNPP63xdQ8PDyxfvhzLly93UIsazp01MkRERA7T6GpkXJ27qnz6NYMMERGR7Bhk7EzqkWGxLxERkewYZOzMnTUyREREDsMgY2dqaUE89sgQERHJjUHGzrhFARERkeMwyNgZh5aIiIgch0HGzjj9moiIyHEYZOzMnSv7EhEROQyDjJ1Jey0xyBAREcmOQcbO3FXWdWRYI0NERCQ3Bhk7U7NGhoiIyGEYZOyMNTJERESOwyBjZ5x+TURE5DgMMnbG6ddERESOwyBjZxxaIiIichwGGTvj9GsiIiLHYZCxM9bIEBEROQ6DjJ1VrCPDHhkiIiK5McjYmZo1MkRERA7DIGNnrJEhIiJyHAYZO+P0ayIiIsdhkLEza5AxsdiXiIhIdgwydqZWsUaGiIjIURhk7EyltNbIsEeGiIhIbgwydsYaGSIiIsdhkLEzDi0RERE5DoOMnUk9MlwQj4iISHYMMnZWsY4Ma2SIiIjkxiBjZ6yRISIichwGGTtTM8gQERE5DIOMnbmXF/tyQTwiIiL5McjYWeW9loRgmCEiIpITg4yduSsrbqnJwiBDREQkJwYZO7MOLQGskyEiIpKbU4NMTEwM+vfvD19fXwQFBWHSpElISkqyOWfEiBFQKBQ2jzlz5jipxTdmHVoCgFITe2SIiIjk5NQgExsbi+joaMTHx2Pr1q0oLS3F6NGjUVBQYHPeo48+ivT0dOnxxhtvOKnFN6ZSVvTIlLBHhoiISFYqZ374r7/+avN81apVCAoKQmJiIoYNGyYd9/LyQkhIiKObVy8KhQJqNyVKzBYOLREREcmsUdXI6PV6AIC/v7/N8dWrV6N58+bo3r07Fi9ejMLCwmqvYTQaYTAYbB6O5u7G/ZaIiIgcwak9MpVZLBbMnz8fQ4YMQffu3aXj999/P1q1aoWwsDAcOXIEixYtQlJSEtavX1/ldWJiYvDyyy87qtlVclcpgRIzSrmWDBERkawaTZCJjo7GsWPHsHv3bpvjs2fPlv7co0cPhIaGYuTIkTh79izatWt33XUWL16MhQsXSs8NBgPCw8Pla3gVuE0BERGRYzSKIDN37lz89NNP2LVrF1q2bFnjuQMHDgQAJCcnVxlkNBoNNBqNLO2sLW5TQERE5BhODTJCCDz55JPYsGEDdu7ciTZt2tzwPYcOHQIAhIaGyty6+lOxRoaIiMghnBpkoqOjsWbNGmzatAm+vr7IyMgAAOh0Onh6euLs2bNYs2YNxo8fj4CAABw5cgQLFizAsGHDEBkZ6cym10japoDryBAREcnKqUFmxYoVAMoWvats5cqVmDVrFtRqNbZt24Zly5ahoKAA4eHhmDJlCl544QUntLb2WCNDRETkGE4fWqpJeHg4YmNjHdQa+1FzaImIiMghGtU6Mk0Fe2SIiIgcg0FGBlKNDNeRISIikhWDjAzcVWW31cQeGSIiIlkxyMiANTJERESOwSAjA5WSQ0tERESOwCAjA+vQUqmJPTJERERyYpCRAXe/JiIicgwGGRlwryUiIiLHYJCRAadfExEROQaDjAy4IB4REZFjMMjIwF1VXiPDYl8iIiJZMcjIwFojY7JwaImIiEhODDIyqKiRYY8MERGRnBhkZKBy49ASERGRIzDIyIDTr4mIiByDQUYGFbOWWCNDREQkJwYZGbBGhoiIyDEYZGTALQqIiIgcg0FGBmoVa2SIiIgcgUFGBlKNjIk1MkRERHKqV5D5/PPP8fPPP0vPn3vuOfj5+WHw4MG4ePGi3RrnqqQgY2GPDBERkZzqFWRef/11eHp6AgDi4uKwfPlyvPHGG2jevDkWLFhg1wa6ItbIEBEROYaqPm9KTU1F+/btAQAbN27ElClTMHv2bAwZMgQjRoywZ/tcEoeWiIiIHKNePTI+Pj7Izs4GAPz222+4/fbbAQAeHh4oKiqyX+tcFHe/JiIicox69cjcfvvteOSRR9C7d2+cPn0a48ePBwAcP34crVu3tmf7XJJ1aInryBAREcmrXj0yy5cvR1RUFK5cuYLvv/8eAQEBAIDExERMmzbNrg10ReyRISIicox69cj4+fnhgw8+uO74yy+/3OAGNQUV68iwRoaIiEhO9eqR+fXXX7F7927p+fLly9GrVy/cf//9uHbtmt0a56oqin3ZI0NERCSnegWZZ599FgaDAQBw9OhRPP300xg/fjzOnz+PhQsX2rWBrkiafs11ZIiIiGRVr6Gl8+fPo2vXrgCA77//HnfeeSdef/11HDhwQCr8vZmpufs1ERGRQ9SrR0atVqOwsBAAsG3bNowePRoA4O/vL/XU3MxU5UHGbBEwWxhmiIiI5FKvHplbbrkFCxcuxJAhQ7B371588803AIDTp0+jZcuWdm2gK7IOLQFlM5fclG5ObA0REVHTVa8emQ8++AAqlQrfffcdVqxYgRYtWgAANm/ejLFjx9q1ga7IWuwLcAo2ERGRnOrVIxMREYGffvrpuuNLly5tcIOaAtsgw6ElIiIiudSrRwYAzGYzvv/+e7z66qt49dVXsWHDBpjN5jpdIyYmBv3794evry+CgoIwadIkJCUl2ZxTXFyM6OhoBAQEwMfHB1OmTEFmZmZ9m+0QbkoF3JTcOJKIiEhu9QoyycnJ6NKlCx588EGsX78e69evx4wZM9CtWzecPXu21teJjY1FdHQ04uPjsXXrVpSWlmL06NEoKCiQzlmwYAF+/PFHrFu3DrGxsUhLS8Pdd99dn2Y7lLRNAdeSISIiko1CCFHnsY/x48dDCIHVq1fD398fAJCdnY0ZM2ZAqVTi559/rldjrly5gqCgIMTGxmLYsGHQ6/UIDAzEmjVrcM899wAATp06hS5duiAuLg6DBg264TUNBgN0Oh30ej20Wm292lUfPf69BXnFJvz+9HC0DfRx2OcSERE1BbX9/q5XjUxsbCzi4+OlEAMAAQEBWLJkCYYMGVKfSwIA9Ho9AEjXTUxMRGlpKUaNGiWd07lzZ0RERFQbZIxGI4xGo/TcWdPBrWvJmDj9moiISDb1GlrSaDTIy8u77nh+fj7UanW9GmKxWDB//nwMGTIE3bt3BwBkZGRArVbDz8/P5tzg4GBkZGRUeZ2YmBjodDrpER4eXq/2NJS14JdDS0RERPKpV5C58847MXv2bCQkJEAIASEE4uPjMWfOHNx11131akh0dDSOHTuGtWvX1uv9VosXL4Zer5ceqampDbpefancWOxLREQkt3oFmffeew/t2rVDVFQUPDw84OHhgcGDB6N9+/ZYtmxZna83d+5c/PTTT9ixY4fNgnohISEoKSlBbm6uzfmZmZkICQmp8loajQZardbm4QzcpoCIiEh+9aqR8fPzw6ZNm5CcnIyTJ08CALp06YL27dvX6TpCCDz55JPYsGEDdu7ciTZt2ti83rdvX7i7u2P79u2YMmUKACApKQkpKSmIioqqT9MdRtoBmz0yREREsql1kLnRrtY7duyQ/vzOO+/U6prR0dFYs2YNNm3aBF9fX6nuRafTwdPTEzqdDg8//DAWLlwIf39/aLVaPPnkk4iKiqrVjCVncleVT79mkCEiIpJNrYPMwYMHa3WeQqG48UnlVqxYAQAYMWKEzfGVK1di1qxZAMpWC1YqlZgyZQqMRiPGjBmDDz/8sNaf4SxSjwyLfYmIiGRT6yBTucfFXmqzhI2HhweWL1+O5cuX2/3z5eTOGhkiIiLZ1XuLAqqZmjUyREREsmOQkYk7p18TERHJjkFGJhxaIiIikh+DjEw4/ZqIiEh+DDIy4dASERGR/BhkZCLttcQgQ0REJBsGGZm4q6zryLBGhoiISC4MMjLh9GsiIiL5McjIhDUyRERE8mOQkQlrZIiIiOTHICMTa5AxcR0ZIiIi2TDIyEStYo0MERGR3BhkZKJSltXIcGiJiIhIPgwyMuEWBURERPJjkJFJxToy7JEhIiKSC4OMTNScfk1ERCQ7BhmZcPo1ERGR/BhkZMLdr4mIiOTHICMTFvsSERHJj0FGJmpVWY2MiT0yREREsmGQkYlKaa2RYY8MERGRXBhkZMIaGSIiIvkxyMjEOrTEIENERCQfBhmZSD0yXBCPiIhINgwyMqlYR4Y1MkRERHJhkJEJa2SIiIjkxyAjEzWDDBERkewYZGTiLq0jw6ElIiIiuTDIyKTyXktCMMwQERHJgUFGJu7KiltrsjDIEBERyYFBRibWoSWAdTJERERyYZCRiXVoCQBKTeyRISIikgODjExUyooemRL2yBAREcmCQUYmCoWCU7CJiIhk5tQgs2vXLkyYMAFhYWFQKBTYuHGjzeuzZs2CQqGweYwdO9Y5ja0Hdzfut0RERCQnpwaZgoIC9OzZE8uXL6/2nLFjxyI9PV16fP311w5sYcO4q9gjQ0REJCeVMz983LhxGDduXI3naDQahISEOKhF9lWxTQGLfYmIiOTQ6Gtkdu7ciaCgIHTq1AmPP/44srOzazzfaDTCYDDYPJyFNTJERETyatRBZuzYsfjiiy+wfft2/N///R9iY2Mxbtw4mM3mat8TExMDnU4nPcLDwx3YYlsq1sgQERHJyqlDSzdy3333SX/u0aMHIiMj0a5dO+zcuRMjR46s8j2LFy/GwoULpecGg8FpYUbapoDryBAREcmiUffI/FXbtm3RvHlzJCcnV3uORqOBVqu1eTiLO4eWiIiIZOVSQebSpUvIzs5GaGios5tSK2oOLREREcnKqUNL+fn5Nr0r58+fx6FDh+Dv7w9/f3+8/PLLmDJlCkJCQnD27Fk899xzaN++PcaMGePEVtcee2SIiIjk5dQgs3//ftx6663Sc2tty8yZM7FixQocOXIEn3/+OXJzcxEWFobRo0fjlVdegUajcVaT60SqkeH0ayIiIlk4NciMGDECQlT/Jb9lyxYHtsb+pAXxTOyRISIikoNL1ci4GmuNjMnCIENERCQHBhkZqZQcWiIiIpITg4yMOLREREQkLwYZGXH3ayIiInkxyMiIey0RERHJi0FGRpx+TUREJC8GGRlxQTwiIiJ5McjIyF1VXiPDYl8iIiJZMMjIiDUyRERE8mKQkZE1yBjZI0NERCQLBhkZNfNWAwCyC0qc3BIiIqKmiUFGRoG+ZZtbXskzOrklRERETRODjIwYZIiIiOTFICOjoEpBpqZdvomIiKh+GGRk1NynLMiUmC0wFJmc3BoiIqKmh0FGRh7ubtB6qAAAV/KLndwaIiKipodBRmbWOpks1skQERHZHYOMzIJ8PQCw4JeIiEgODDIy48wlIiIi+TDIyMxeQaa41MyZT0RERH/BICMzewSZtNwi9H1lK+79JB76olJ7NY2IiMjlMcjILNCn4cW+Ry/rUVBixt7zObj/v/HI4ZYHREREABhkZBekbXiPTG5hRXA5nmbAfZ/EIcvA6dxEREQMMjKThpbyGxJkyoaT+rduhmCtBqcz8zH14zhczi2ySxuJiIhcFYOMzKxDSzkFJSg1W+p1jWvlQaZHCz+se2wwWjbzxIXsQkz9KA5XGxCQiIiIXB2DjMyaeanhplQAALLz61fbYh1aaubljogAL3z7WBQi/L1wObcIGw9etltbiYiIXA2DjMyUSgWa+6gBAFl59atrsQ4t+XmXXSfMzxP39g8HABxMzW14I4mIiFwUg4wDNHR132vlPTJ+nu7Ssd7hfgCAQym5DWobERGRK2OQcYCGriVj7ZFp5qWWjkWG+0GhAC7nFnEGExER3bQYZBzAWvDb4B4Zr4oeGR+NCp2CfQFweImIiG5eDDIO0JAp2EII5Jav5ls5yABA7wg/AMBBDi8REdFNikHGAayL4mUZ6h5kikrNKDGVTduuPLQEAL3DmwEADqZca2ALiYiIXBODjANIQ0v16JGxriGjdlPCS+1m85q1R+bIJT1M9VyjhoiIyJUxyDhAQ4p9rWvI6LzcoVAobF5rF+gDX40KRaVmJGXmNbyhRERELsapQWbXrl2YMGECwsLCoFAosHHjRpvXhRB46aWXEBoaCk9PT4waNQpnzpxxTmMboHKQEULU6b0VM5bcr3tNqVSgF+tkiIjoJubUIFNQUICePXti+fLlVb7+xhtv4L333sNHH32EhIQEeHt7Y8yYMSgudq3pxtYgU1RqRr7RVKf3VsxYUlf5unU9GQYZIiK6Gamc+eHjxo3DuHHjqnxNCIFly5bhhRdewMSJEwEAX3zxBYKDg7Fx40bcd999jmxqg3ipVfDRqJBvNOFKnhG+HhW9K8WlZqzYeRbjeoSgc4j2uvdaa2QqL4ZXWe+IsoLfQ6ks+CUioptPo62ROX/+PDIyMjBq1CjpmE6nw8CBAxEXF1ft+4xGIwwGg82jMaiuTua7xEt4d/sZvLUlqcr36aV9lqrukelV3iNz9koB9OWhh4iI6GbRaINMRkYGACA4ONjmeHBwsPRaVWJiYqDT6aRHeHi4rO2srepmLu2/kAMAuHStqMr3ST0y3lX3yDTzVqN1gBcA4NClXHs0lYiIyGU02iBTX4sXL4Zer5ceqampzm4SgOp7ZA6U17ZkVTOjqWKfpap7ZICK4SWuJ0NERDebRhtkQkJCAACZmZk2xzMzM6XXqqLRaKDVam0ejYE1yFQOLFfyjEjJKQQA5BSUSAvfVaavYdaSFVf4JSKim1WjDTJt2rRBSEgItm/fLh0zGAxISEhAVFSUE1tWP1X1yBz4Sw/K1SoWzLvRrCWgYoXfQ6m5sFjqNr2biIjIlTl11lJ+fj6Sk5Ol5+fPn8ehQ4fg7++PiIgIzJ8/H6+++io6dOiANm3a4MUXX0RYWBgmTZrkvEbXU5VB5qJtkMk0FCPMz9PmWE3ryFh1DvWFRqWEvqgU57ML0C7Qx17NJiIiatScGmT279+PW2+9VXq+cOFCAMDMmTOxatUqPPfccygoKMDs2bORm5uLW265Bb/++is8PDyc1eR6q02PTFV1MhUbRlbfI+PupkRkSx32XbiGgym5DDJERHTTcOrQ0ogRIyCEuO6xatUqAIBCocB//vMfZGRkoLi4GNu2bUPHjh2d2eR6++uspRKTBYcv6QEAHYLKgkeWwXahP4tFSFsU1NQjA3A9GSIiujk12hqZpsa6A3Z2vhFmi8CJdANKTBb4ebljYFt/ANf3yOQVm2AtedHdKMiUrydz4GKuXdtNRETUmDHIOEiAtwZKBWARQHaBEYnl9TF9Ipoh2LdsqCzzLz0y1kJfL7UbNCrbna//qmtY2eyss1fy67yfExERkatikHEQN6UC/t4VdTLW+pi+rZohWFsWZP7aI2Otj6luVd/KrNcwmiwwFNdtPyciIiJXxSDjQJULfq0zlnpH+CGwfNgpy2AbZCqmXtc8rAQAHu5u0Hqoyq/jWptqEhER1ReDjANZg8yRS3qk64vhplSgZ0s/aWgpK882gOTWIcgAQFA1PTtERERNFYOMAwWVB5ktx8v2iuoc4gtvjaqiELigBKXmitV9rWvI1DT1urJga89OHntkiIjo5sAg40DWHpnjaWU7cvdtVTZl2t9LDZVSASFsV/e9VovF8CoLsvbsGNgjQ0RENwcGGQeyriVj1ad87RelUlGxF1OlEJJbiw0jK7P2+GQyyBAR0U2CQcaBrGHFyhpkgMohpGJYqGJoqXY9MhUbU3JoiYiIbg4MMg4UVCnINPfRINy/Yl+lqgp1r0mr+ta2RobFvkREdHNhkHGgyj0yfSL8oFAopOdBUm9K5aGluvXIBFWxnxMREVFTxiDjQJWDjLXQ16qiULdiWKhiHZla1shoq14hmIiIqKlikHEgH40Knu5lWw30+UuQqZg6XdGboq/zrKWyaxSWmJFv5Oq+RETU9Kmc3YCbiUKhwNOjO+LslXybQl+gYlNJa29KqdmCvPIwUtseGW+NCj4aFfKNJmQZiuET6GPH1hMRETU+DDIO9sjQtlUel4aWyntkrPUxCgWg86xdj0zZdTRlQSbPiLYMMkRE1MRxaKmRsPbIXM03wmS2QF9UVh+j9XCHm1JR01ttBFYxjZuIiKipYpBpJAK8NXArX903u6BEWtW3tjOWrKwFv5y5RERENwMGmUbCTalAc5+yWpgsgxHXCuo2Y8kquIpp3ERERE0Vg0wjYq2TyTQUSzUytZ2xJF1Dy6ElIiK6eTDINCKVp2DnFtVtVV8rbhxJREQ3E85aakQCK/XIlJgtAOo2YwmovEIwe2SIiKjpY5BpRGwXxRMA6tEjw/2WiIjoJsIg04hU3qZArSob9WvmXb8ambxiE4pKzPBUu9m3kURERI0Ia2QakcobR1r3Warr0JKvRgUPd2X5dTi8RERETRuDTCMSLA0LVZ61VLehJYVCcd0qwURERE0Vg0wjYh0WupJnRHZB/WYtAZVqbThziYiImjgGmUYkwFsNpQKwiIqVeeu6si9gux4NERFRU8Yg04io3JQI8NHYHKtPkAnk6r5ERHSTYJBpZKzDQgCgUirgo6n7xLIgbcPWksk3mqQtEoiIiBozBplGxjosBJT1xigUtd/52irYt/4bRwohMPWjONz69k7kMMwQEVEjxyDTyFinYAN13zBSukYD9ls6nmbAiXQDcgtLEXc2u16fT0RE5CgMMo2MdWVeoO4bRkrXaMD06+0ns6Q/77uQU6/PJyIichQGmUamco+MzrOePTLl18gtLIXRZK7Te7efypT+nHjxWr0+n4iIyFEYZBqZYDv0yPh5uUPtVvafti51MlmGYhy5pJeeH0/TI99oqlcbiIiIHKFRB5l///vfUCgUNo/OnTs7u1myqtwj08y7fj0yCoVCmoKdWYdF8X4/VTas1LOlDi2becIigEMpufVqAxERkSM06iADAN26dUN6err02L17t7ObJKsgbeWhpfr1yFS+zpU6TMHeVl4fM7JLMPq39gfAOhkiImrcGv3u1yqVCiEhIc5uhsM099FAoQCEqN/2BFZBdVwUr7jUjN3JVwAAI7sE4VBqLjYcvIz9FxlkiIio8Wr0PTJnzpxBWFgY2rZti+nTpyMlJaXG841GIwwGg83Dlbi7KRFQPqRU3xoZoNLMpVoOLf159iqKSy0I03mga6hW6pE5mJKLUrOl3u0gIiKSU6MOMgMHDsSqVavw66+/YsWKFTh//jyGDh2KvLy8at8TExMDnU4nPcLDwx3YYvvoFOILAGgT6F3vawTXcS0Z67Tr27oEQaFQoH2gD3Se7igsMeNkumuFQSIiunk06iAzbtw4/O1vf0NkZCTGjBmDX375Bbm5ufj222+rfc/ixYuh1+ulR2pqqgNbbB/vT+uDjdFD0DlEW+9r1GUtGSGEVOg7snMwAECpVKBvq2YAgH0XOA2biIgap0YdZP7Kz88PHTt2RHJycrXnaDQaaLVam4er8fdWo1e4X4OuEaitfY3M8TQD0vXF8HR3Q1S7AOl4v9ZlQWZ/FQW/RpMZWdxdm4iInMylgkx+fj7Onj2L0NBQZzel0bMW+9Zm1pJ1WOmWDs3h4e4mHa+YuXQNQgjpuMlswX2fxGPA69uxbNtpWCwCREREztCog8wzzzyD2NhYXLhwAX/++ScmT54MNzc3TJs2zdlNa/SsC+tdzS+5YbHu7+Wr+Y7qEmRzvEcLHdRuSlzNN+JidqF0/Mv4izhYvr7Msm1nMPvLRBiKS+vcxrziUny7PxUvbDyK81cL6vx+IiKiRh1kLl26hGnTpqFTp06YOnUqAgICEB8fj8DAQGc3rdHz91JDpSzbOftqfvXDS1mGYhwuX8331s62QcbD3Q2RLXUAKtaTyTIU4+3fTgMA7owMhVqlxLaTmZj0wR4kZ1VfhG1ltgjsOn0F89YeRP/XtuG5747gq/gUTPxgN2JPX6n7X5SIiG5qjXodmbVr1zq7CS5LqVSguY8GGYZiZBmMCNV52rxuDRT//eMcAKBnuJ9UIFxZv9b+2H/xGvZfuIa/9QvHqz+fRL7RhJ7hfnj3vt44nqbHnC8Tce5qASZ+sAfLp/fBiE5B110HAAzFpZj6URxOZVQEnraB3vBSu+HYZQMeWrkX/xjfBQ/f0gYKhaLKaxhNZiScy8GOpCy4KRR46JY2aOHnWeW5JrMFJWYLvNSN+seciIgagL/hm7AgbVmQsU7BLi41I0NfjF+OpWNNQgouXSuSzn34ljZVXqN/62b4KBbYdzEHu89cxQ+H06BUAK9N6g43pQKRLf3ww5O3YO6aA4g/l4Onvz2MPc/fZlNrY7V2bwpOZeTBV6PCpN4tMKVvS/RsqUOJ2YKXNh7HN/tT8erPJ3Ei3YDXJ/dAUYkZV/ONuJJnxMWcQuxMysIfZ66isKRiI8wv4i9i1uDWeGJEO/iVLyCYri/C13tT8fXeFBQaTfji4QHo28rfnreWiIgaCYWoXMXZBBkMBuh0Ouj1epecwdQQj3y+H9tOZiLQV4MSkwX6Its6Fp2nO+7p2xL3D4xAu0CfKq+RW1iCXv/ZCgBo2cwTl64VYdbg1vj3Xd1szis1WzDizZ24nFuE1yZ3x/SBra57ffgbO5CmL8YbUyIxtb/t+j5CCHz+5wW88vNJmC1CWt24KoG+GtzWKQgXcwoQf65syEvrocJDQ9ogKSMPW09mwlypADnIV4Ofnrqlyh6nv7bhcm4RTqXnoVOIL8L9vWo8n4hITgdSruG7xEt4dnSneu+958pq+/3NHpkmrGOwD7adzLTZAVujUqJbmBbTBkRgQs+wKntOKvPzUqNjsA9OZ+bj0rUiNPfRYOHojted5+6mxCND2+DlH0/gv7vO4b7+EXBTVgwPbT6WgTR9MZr7qHFXr7Dr3q9QKDBrSBt0CPbF3DUHcK2wLHTpPN3R3EeNIF8PDGzrj5Gdg9EtTAulUgEhBHYmXcGSzaeQlJmHd7efka43oI0/7h8QgeU7knEmKx9zVx/E6kcHwt3Ntizs7JV8fJ94CUcv63Hssl763CBfDWKfvRWe6prvDzVeabllP69qVaMuBSSqUlGJGdGrDyBdXwyNSol/Teh24zfdpBhkmrC5t7VHr3A/eKlVCNZqEOTrAa2nqtr6k+r0a+2P05n5AIAX7+wCrUfVWydM7ReOZdvO4EJ2IbaeyMDY7mXT5IUQ+LS8FmfGoFY1hqch7Zsj/h8jcTW/BM191NCoqj9XoVDg1s5BGNYxEBsOXsb6A5fQNtAbDwxqLa2O3KOlDpM+2IO9F3Lw+i8npV8GFovAZ3vO440tSSgxVczqUikVULkpkJVnxJq9KdUOud1MjKayobya/ls0JkIIvP3baXywIxm3dgrEyocGOLtJZCdCCLyxJQmHU3Mxb2QHDGwbcOM3uaj//XEO6fqysoB1+y9h4e0d4VvN796bHYNME+alVmF0t4ZvuHlbpyCsSUjB0A7NcVfP63tTrLw1KjwY1Qrv/56MFbHnMKZbCBQKBRIvXsPhS3qoVUrMGNSq2vdbaVRu1RbwVsVNqcA9fVvinr4tr3utXaAP3p7aE7O/TMTKPRfQK9wPfSKa4Zl1h5FwvmxY6pb2zTG2ewh6tNChU4gvNhy8jMXrj+Kj2LOYPjDihr1WTcnJdAPWJKQgJacQmeX1VdcKS+HrocKm6CFoW80QZGNhNJnx7Loj+OFwGgBgR9IVxJ3Ntlno0Vmy8oqRllvc4MUub2bf7k/Fip1nAQB/ns3G3b1bYPH4LggsXzerqcgyFGNFbNnf00vthnyjCev2X8Lf+Q+rKrFGhm5ICIHDl/ToHOJ7wy/1q/lGDF7yO0pMFnwzexAGtg3AnC8T8evxDNzXPxxLpkQ6qNW23tqShA92JMPDXQk3hQIFJWZ4qd3wwh1dMW1AuE0vVYnJglvfKqv3+feErpg1pOn/8kjKyMO720/jl6MZ1Z4zsnMQPp3V34GtqptrBSV47MtE7L2QA5VSgW4tdDicmotBbf2xdnaU09s29t1dyDQYMWNQBF66s9tNP+S1/0IOXvnpBCwCUKuUULspoXFXokcLHZ68rcN19ycpIw8Tl+9GcakF/Vo1Q2LKNQgB+Hqo8MzoTpgxqJXNcPZfCSHq3BvtLM99dxjf7r+EXuF+uKdvS7yw8Rgi/L2w45kRNf4da8NktuDno+n4dPd5GIpKsXx6H3QL09mp5fbFGhmyG4VCUet/RTb30eBvfVtidUIKPt51DqE6T/x2ouzL0Zn/mlhwe0ccuazHrvK1avq3boa3/tYTrQKu35hTrVLi8RHt8MLGY1gRexb3Dahfr4wQAtcKS3ExuwApOYW4mF3WyzGgjT/GdQ+t8ovs/NUCrD9wCbnltToKBaBA2X2dNaR1nbqWt5cXPd/eNbjaX+DJWXlYuu0MfjmaLhVX39EjFMM6Nkew1gPBWg+UmCyYsuJPbD+VhV2nr2BYx8a3jtPF7AI8tHIfzl0tgK9GhRUz+qJtoDeGv7kD8edyEH8uG4OcNAwhhMALG48hs3wn+q/iU5CUkYcPp/d1aE+CxSIggAZ/EdqD2SLwzw3HkJR5/dpTO5OuYN+FHHw8ox90XmU/74UlJsxdcwDFpRYM7xiIlbP648hlPV7ceAxHL+vxrx+OY/3By1g6ted1vYZCCGw4eBmv/3IS3cJ0+OD+3o1iiOZidgESL17D+B6hNr9fjl3WY13iJQDAi3d2RddQLd7ckoSUnEL8fioLt3cNrtfnFZaU9er8949zNjNW7/04Hp880BeD2zev1XUMxaXwULk1qiDOHhmyu/NXC3Db2zshBDC8YyBiy7/8vvi7c2sVcgtLEPPLKXQO9cWDUa1r/IVuNJkx4s2dSNcX45WJ3fBAVOs6fZbRZMbsLxKrXeQvWKvBg1Gtcf+ACOg83bHrzBWs+vMCdiZVvyhg5xBfrHyo/3VrAlVlR1IWHlq5DwAwsI0/XpvcHe2DfKXXcwtLsHTraXyVkCLN8BrfIwRPjexQ5Wal//nxBD7bcx4dgnywed5QqNwazy+x/RdyMPvLROQUlKCFnyc+m9VfqpH654ajWJ2Qgqi2Afh69qBaX7O4tGzq/9X8EmTnG9E20Adtmle/G31RiRkWIeCtuf7fhhsPXsb8bw5BpVTg2TGd8MHvycgzmhCi9cDHD/RFTxmHmlJzChF7+gr+OHMFfyZnQ+vpjrWzBzl9Rt6mQ5cxb+0h+HqosHRqL5gsAiVmC7LzjXj7t9PIN5rQPsgHK2f1R7i/l9RDEeSrweZ5QxHgUxYAzRaBNXtT8Mavp5BXbIKHuxL/GN8FMwa2glKpQFZeMf6x/hi2ncyUPrtHCx1WPdRfukZNfj2Wjpd/PIFXJ3XHyC5VB4hzV/Lx2s8n8dCQNrilQ+3CQF5xKUYv3YV0fTFaBXjh1UndMbRDIIQQuP+/CYg7l40JPcPw/rTeAIAlm0/ho9izdf45tvrxcBpe2nRMmszg763GzKjWiDt3FfHncqB2U+Kde3vizsjqSwcOp+biw53J+O1EJgK8NXhqZHvc1z9C1kBT2+9vBhmSxeNfJWLzsYphii/+PqBR/ku+Jl/EXcBLm44jVOeBnc+OqFOx6782HcPncRcBACFaD0QEeKGVvxd8Pdzx45E0aSaZh7sSgb4apOaU/QtJoQBu7RSE7i100vxziwC+2Z+KK3lGhGg9sPKh/ugSWv3Pck5BCcYs22UzW83dTYHHhrXDnBHt8H3iJbyz9bQ0HX9Ul2A8PbpjjdfUF5ZixFs7cK2wFP+Z2A0P1iLYFZWY4aZUyPqLbtOhy3h23RGUmC3o0UKHT2f2Q5C2Ypr95dwijHhzB0rNAt8+FoUBbWpeT2j9gUt45acT0i98K2+1GzbNvQXtg66vEbqSZ8TkD/cgp6AEr0zsjimVarXScoswZtku5BWb8PTtHfHkyA44eyUfs7/Yj7NXCqBWKfHmPZGY2KtFrf/O+y7k4PdTWfD1UCHQR4NAXw2a+2hQWGLGhasFOHe1AOev5uNURp7N1iJW7QK98f3jg6V1l+qquNQMhaL+xd8mswW3L92F81cLpHtS2Yk0A/6+ah8yDMVo7qPBff3D8cGOZCgVwOpHBlVZ75SWW4RnvzuMPcnZAIChHcrq3t7ckoTcwlK4uynw0JA2+D7xErILStAu0BtfPjwQYTXU4umLSnHbWzuRXVA28WD7whFSD5GV2SIw+cM9OHJJD52nO7bMH4YQXc3LPADAv384jlV/XrA5NqlXGAa1DcDz649CrVLi96eHo2UzL+nvN/SNHTBbBDbPG1rj/6uVlZgseP2Xk9JnRfh74dFhbXFPn5bwVLuhuNSMhd8ewi9HM6BQAP+603Yo3WwRiD+XjQ93Jkv3trIIfy88PbojJkSGQSlDTx+DTDkGGec4mHINkz/8E0DZNPAt84e5zPi0VXGpGcPf3IFMg7HKtXGq89ORNMxdcxAAsHJW/+u2fjCazPj5SNkY9fE0AwDAV6PC1P7heDCqVZXDXZeuFWLWyn1IzsqHj0aFFTP6YGiH64OhEAJzvkrEluOZ6BDkgxUz+uD1X07h91NlG4Oq3ZQoKd97q3OIL166s2utu5S/jLuAFzcdh5+XO2KfufW6X+pWB1Ou4bM9F7D5aDo81W64MzIM9/RtgT4RzWx+BiwWgTR9EXSe7nXu6hdC4N3tZ7BsW9mU+zHdgrH03l5VruL8jw1HsSYhBUPaB2D1I9X/a/ZMZh7ueH+3NItN7aZEcx81TBaBrDwjOgT5YGP0EJtel1KzBdP/l4C95yt2iP9b35Z4eWI3eKjcMOPTBPx5Nhu9I/yw7rEoqScrr7gUC745LPUUvHRn1xsOvR5OzcXbW09Lw6O1oVIq0CeiGYZ2aI5eEX5Y9N0RpOmL0a9VM3z1yMAbDplezi3Cyz8cx7mrBdAXlUJfVIoSkwWe7m544c4uuH9AxHX/X5eaLXhrSxK2HM/AyxO7Y/hf/gHz7b5UPPf9Efh7q7HruVvhU0UvVrq+CA+t3GezCvj8UR0wf9T1Sz9YWSwCX8RdQMzmUzBWmonYo4UOb/2tJzqF+OLslXw88L8EpOmLEabzwFePDKy2gP2vYWPagAjE3N3D5pz//XEOr/58Uno+tENzfP7QgBq/1A+l5mLyh3sgBPDRjD6IP5eDz+Mu2Kyb9cSIdnhubGeb90WvOYCfj6Rjar+WeOOentVe3+pybhGiVx/AodRc6ZoLb+94XW+q2SLw8o/H8UX5P7yaebmjxGSB0WSBqdJ6XCqlAhN7tcAjQ9tg/8VreG/7GekfS11Ctfjn+C617pGqLQaZcgwyzjP9f/HYk5yNt/7Ws8oZRa5g5Z7zePnHE2jh54mPH+gLo8kCo8kMY6kFrZt7XzfccP5qASa8vxv5RlOVv4wqE0Jg/8VryDQU49ZOQVUOS1SmLyzFY1/tR/y5smLWlyZ0xfSBtgWO6/an4tnvjsDdTYENTwxB9xY6CCGw5Xgm/v3DcWQYitHMyx0LR3fCtP7hdRoiMpktGP/eHzidmY+HhrS2WdeiuNSMLcczsHLPBekX51+1ae6NUV2CcCXPiOQr+TibVYCiUjM83JWYPrAVHhvW1qY3xXp/1u5NxaVrhfD3VqOZtxr+XmqcycrDluNlIeCx4W2xaEznar88Ll0rxK1v7USpWWDdnChpV/e//t3uXvEnjlzSY3jHQLx/f2/4asqWKriSZ8Qd7/2BrDwjJvYKw7J7e0lf3i//eBwr91yAr0aFe/uH47M952ERQIcgHwzvGIj/7T4PT3c3/DJv6HU/KxaLwKs/n8Rne84DAJ4a2QELRnW4LhicSDNg6bbT2Hqi7O+rUipwR2QoVEolrpSvfH013wiNSok25T+T1kffVs1sQuLpzDzcs+JPGIpNGNstBMun96l2iPVKnhFTP46rcUPXqf1a4j8Tu0uB6HJuEeauOSBtKuvhrsTqRwZKK2sbTWbc9lYsLucW4Z/ju+DRYW2rvXZecSmi1xzErtNXENU2AF89MrBW9T3JWfl4et1hnEwz4Mnb2mPOiHY260ddzi3CA/9LwLmrBQjwVuPLhweia5jtd0NSRh7Gv/cHzBaBeSM7SGtUfTcnCv3Kf35ScwoxeukuFJWa8diwtvg87gKKSy01htJSswUT3t+NUxl5uLt3C7xzby8AZSF18fqjOJFuQHMfDXY+O+K6gJd4MQdTVsRBrVIi7vnbahwas+5nd62wFFoPFZbe26vaoTGg7P+15TuS8Vb5PnqVebgrcV//CDwytI3UQwSU1dys3HMBH8WeRV6xCS/f1Q0zB7eu9jPqg0GmHIOM8+QWluB4mgGD2wW4XG+MVXGpGUPf2GEzTGOlUJQVxs4f1QHtg3xRXGrG5A//xMl0Awa08ceaRwbavZbEaDLjue+OYNOhsunF3cK0+NeEbhjQxh+pOYUY9+4fyDea8NzYTnhiRHub9+YbTdh95iqi2gZU25tyI3+cuYIHPt0LlVKBVyZ1R3JWPhIvXsPxND1KzWW/StRuSkzoGYaHhrSGobgU3ydexuZj6TZbS1i5KRVSjY5apcS0/uG4f2Ar7Em+iq/3puBMVn61bVEpFXh1UnfcNyDihu1evP4Ivt6bilvaN8dXjwy87vX3t5/B21tPQ+uhwm8Lhl83PLDvQg7u+yQeZouQaqastS8A8MkDfTG6Wwjiz2Xjqa8PIqvSz8srk7rjgWqWHRBC4IPfk/H21rIvkAejWuHfE7qhoMSEn46k49v9qVIoUCqAyb1bYt7IDogIqH+NS/y5bDz46V6UmC2YGdUK/76r23X/f+oLS3HvJ2X7orXw80TM3T0Q4KOGztMdWk93rI5PwZtbTsEiyno8VszogzNZ+VjwzSHkln95tg/ywYGUXGg9VPh2ThQ6h2il4dogXw12PXfrDXuETGYLEi9eQ89wvzoV3AshYDRZqn3P1XwjZn62F8fTDGjm5Y41jw6Shmsq16mM6RaMjx/oh+e/P4K1+1LRIcgHPz81FO5uCjz42V78ceYqBrbxx9ePDsLqhIt4cdNxqFVK/PTkLegY7Hvd534UexZLNp9CMy93bFs43CaMmMwWbDuZhU4hvlXWYwkhMHF52TBWVUNyVnFns/HgZwkoNQt0b6HFiul9a10TlWkohqGoFGqVEhqVGzQqJbw1qhqHh3MLS/Bl3EU8Nryd3YeRGWTKMchQQ206dBmv/nwSCpTtCK5RKeGmVEjd3goFMLFnGCwC+OFwGgK81fhl3lAEa288Vl4fQgis3HMBS7edRl6xCUBZoMowFCPx4jX0b90Ma2dHyTY75eFV+7C9fKiqslCdB+7rH4H7B0ZcNxunwGjCr8cykJhyDS38PNEhyAftg3wQ4e+F3clX8f7vyUi8eO26a3q6u+GunmEY3D4AhqJS5BSUIqfAiKJSM+7pG37Dmher1JyyXhmTReCJEe3w1MgO0pfc8TQ9Ji3fg1KzwNJ7e2Jy76p7D63DCO5uCrwysTv+/eNxFJda8ORt7fH06E7SeVfzjVjwzSH8ceYqbu0UiM9m9b9hkP8y/iJe2nQMQgDdW2iRnJWP4tKy4RE3pQLjuodg/qiOVdbo1Efl4c87I0Px2LB26FG+032B0YQZnybgYEouAn01WPdYFFpX8cW6+8xVPPl12SrcvhoV8oxlP4uRLXVYfn8fBPio8cCne5F48RqCfDX46pGBmP6/BFzJM9argN7e9EWleODTBBy5pIe/txprHh2IziFa/HwkHdFrDkCjUmLbwuEI9/dCbmEJRr4di+yCEjw7phNCdR5Y+O1hqFVK/DpvKNoG+kAIgYdW7cPOpCvoEqrFxujBNnVEKdmFGL0sFsWllnr3UlvDs9ZDhS8eHnjdbNLkrDzc/WFZj9u47iFYem8vl14Hi0GmHIMMyeVkugHLtp2WhjiAslDz5d8H2n2suCrZ+Ua8vfU01u5NgXUo21vthl/nD5N1Vop1qrPG3Q39WjVD3/JHy2ae9e55E0Ig7mw23t1+Bgnnc9AlVIv7B0ZgYq+waleSrqv/+/WUtJham+beeG1yd/Rr5Y+7Pijr6h/dNRgfP9C32r+DEAJPrD5gU8Q+vGNZUPlraLRYBI6nGdA51Pe6bTGqs+nQZTz97WGpLqF9kA+m9muJSb1b3HCfsPr4bPd5/OenE9LzAa39MWtIa6xOuIg9ydnQebrjm8cGVTmLzerStUI8/tUBHL2sB1DWo/TPO7pIX+D6wlJM/TgOSZl50KiUMJosaOHniR3PjGgU03f/GmY+ndkP0asPIE1fjHkjO2DB7RU1OdYQoVEp4al2Q25hKZ4d0wnRt1b0fGblFWPssj+QU1CCaQMiMKZbMJQKBRSKst6YPcnZGNwuAKsfGViv/1dKTBZM/TgOh1Jz4enuhg9n9MGtncpq8K7mlxWdp+YUoW+rZlhdixqoxo5BphyDDMnt2GU9lm07je2nsvDsmOuHdOR2Mt2AV38+gYRzOXh7as86zYBpjIpKyupm7D0cWVYrlIF//XBcWtOlc4gvTmXkwd9bjS3zh91wXZe84lLc9cEenL9agHB/T/w495Z6z/6pSvy5bOxIysLYbiHoFe4n+5Ds0Ut6fLr7HH46km5T2OmtdsPqRwfVav2o4lIzPv/zAtoH+VRZh5FlKMaUj/6UZua9cU8kpvYLv+48Z9EXluKBz8rCjFJRNkuwhZ8nti0cbrPXmhACD3y6F7uTrwIoK3D9Ye6Q64LqluMZeOzLxCo/S61SYsv8YTVO5b+RAqMJj68+gF2nr0ClVOD/pkTijshQ3PdJPA6l5qJVgBc2PDEE/k1gk0kGmXIMMuQoRpPZqfsROfvzXYWhuBRv/pqErxIuSjNFPpzeB+N7hNbq/RezC/BV/EXcP7BVg76QGpNMQzG+jLuI1QkXUVxqwaez+mFwO/v1Kl7MLsD0/yWguY8G382JalTrEAFlYWbGpwlSz9KK6X0wroqfhwtXCzBm2S6Umi3YGD0EkS39qrzeB7+fwZbjmbAIASEAS/mqwg8NaW2XEFdismDR90ew4eBlABWB3M/LHesfH9zotxKpLQaZcgwyRFSVxIvXsHTrafQM1+HZMdXPLruZlJTPypNj5VuzRUCpQKMt/NcXluLFTccQovPA4nGdq22ntbDd2XtmWSwCMZtP4r9/lM16U7sp8dUjA2tdN+YKGGTKMcgQEVFT9dnu8/g87gKeG9MZd0TWrlfRVTDIlGOQISIicj21/f5uXAOVRERERHXAIENEREQui0GGiIiIXBaDDBEREbksBhkiIiJyWQwyRERE5LIYZIiIiMhlMcgQERGRy2KQISIiIpfFIENEREQui0GGiIiIXBaDDBEREbksBhkiIiJyWQwyRERE5LJUzm6A3IQQAMq2AyciIiLXYP3etn6PV6fJB5m8vDwAQHh4uJNbQkRERHWVl5cHnU5X7esKcaOo4+IsFgvS0tLg6+sLhUJht+saDAaEh4cjNTUVWq3WbtelqvF+Ow7vtePwXjsO77Xj2OteCyGQl5eHsLAwKJXVV8I0+R4ZpVKJli1bynZ9rVbL/ykciPfbcXivHYf32nF4rx3HHve6pp4YKxb7EhERkctikCEiIiKXxSBTTxqNBv/617+g0Wic3ZSbAu+34/BeOw7vtePwXjuOo+91ky/2JSIioqaLPTJERETkshhkiIiIyGUxyBAREZHLYpAhIiIil8UgU0/Lly9H69at4eHhgYEDB2Lv3r3ObpLLi4mJQf/+/eHr64ugoCBMmjQJSUlJNucUFxcjOjoaAQEB8PHxwZQpU5CZmemkFjcdS5YsgUKhwPz586VjvNf2c/nyZcyYMQMBAQHw9PREjx49sH//ful1IQReeuklhIaGwtPTE6NGjcKZM2ec2GLXZDab8eKLL6JNmzbw9PREu3bt8Morr9js1cN7XT+7du3ChAkTEBYWBoVCgY0bN9q8Xpv7mpOTg+nTp0Or1cLPzw8PP/ww8vPzG944QXW2du1aoVarxWeffSaOHz8uHn30UeHn5ycyMzOd3TSXNmbMGLFy5Upx7NgxcejQITF+/HgREREh8vPzpXPmzJkjwsPDxfbt28X+/fvFoEGDxODBg53Yate3d+9e0bp1axEZGSnmzZsnHee9to+cnBzRqlUrMWvWLJGQkCDOnTsntmzZIpKTk6VzlixZInQ6ndi4caM4fPiwuOuuu0SbNm1EUVGRE1vuel577TUREBAgfvrpJ3H+/Hmxbt064ePjI959913pHN7r+vnll1/EP//5T7F+/XoBQGzYsMHm9drc17Fjx4qePXuK+Ph48ccff4j27duLadOmNbhtDDL1MGDAABEdHS09N5vNIiwsTMTExDixVU1PVlaWACBiY2OFEELk5uYKd3d3sW7dOumckydPCgAiLi7OWc10aXl5eaJDhw5i69atYvjw4VKQ4b22n0WLFolbbrml2tctFosICQkRb775pnQsNzdXaDQa8fXXXzuiiU3GHXfcIf7+97/bHLv77rvF9OnThRC81/by1yBTm/t64sQJAUDs27dPOmfz5s1CoVCIy5cvN6g9HFqqo5KSEiQmJmLUqFHSMaVSiVGjRiEuLs6JLWt69Ho9AMDf3x8AkJiYiNLSUpt737lzZ0RERPDe11N0dDTuuOMOm3sK8F7b0w8//IB+/frhb3/7G4KCgtC7d2/897//lV4/f/48MjIybO61TqfDwIEDea/raPDgwdi+fTtOnz4NADh8+DB2796NcePGAeC9lktt7mtcXBz8/PzQr18/6ZxRo0ZBqVQiISGhQZ/f5DeNtLerV6/CbDYjODjY5nhwcDBOnTrlpFY1PRaLBfPnz8eQIUPQvXt3AEBGRgbUajX8/Pxszg0ODkZGRoYTWuna1q5diwMHDmDfvn3XvcZ7bT/nzp3DihUrsHDhQvzjH//Avn378NRTT0GtVmPmzJnS/azqdwrvdd08//zzMBgM6Ny5M9zc3GA2m/Haa69h+vTpAMB7LZPa3NeMjAwEBQXZvK5SqeDv79/ge88gQ41SdHQ0jh07ht27dzu7KU1Samoq5s2bh61bt8LDw8PZzWnSLBYL+vXrh9dffx0A0Lt3bxw7dgwfffQRZs6c6eTWNS3ffvstVq9ejTVr1qBbt244dOgQ5s+fj7CwMN7rJoxDS3XUvHlzuLm5XTd7IzMzEyEhIU5qVdMyd+5c/PTTT9ixYwdatmwpHQ8JCUFJSQlyc3Ntzue9r7vExERkZWWhT58+UKlUUKlUiI2NxXvvvQeVSoXg4GDeazsJDQ1F165dbY516dIFKSkpACDdT/5Oabhnn30Wzz//PO677z706NEDDzzwABYsWICYmBgAvNdyqc19DQkJQVZWls3rJpMJOTk5Db73DDJ1pFar0bdvX2zfvl06ZrFYsH37dkRFRTmxZa5PCIG5c+diw4YN+P3339GmTRub1/v27Qt3d3ebe5+UlISUlBTe+zoaOXIkjh49ikOHDkmPfv36Yfr06dKfea/tY8iQIdctI3D69Gm0atUKANCmTRuEhITY3GuDwYCEhATe6zoqLCyEUmn7tebm5gaLxQKA91outbmvUVFRyM3NRWJionTO77//DovFgoEDBzasAQ0qFb5JrV27Vmg0GrFq1Spx4sQJMXv2bOHn5ycyMjKc3TSX9vjjjwudTid27twp0tPTpUdhYaF0zpw5c0RERIT4/fffxf79+0VUVJSIiopyYqubjsqzloTgvbaXvXv3CpVKJV577TVx5swZsXr1auHl5SW++uor6ZwlS5YIPz8/sWnTJnHkyBExceJETgmuh5kzZ4oWLVpI06/Xr18vmjdvLp577jnpHN7r+snLyxMHDx4UBw8eFADEO++8Iw4ePCguXrwohKjdfR07dqzo3bu3SEhIELt37xYdOnTg9Gtnev/990VERIRQq9ViwIABIj4+3tlNcnkAqnysXLlSOqeoqEg88cQTolmzZsLLy0tMnjxZpKenO6/RTchfgwzvtf38+OOPonv37kKj0YjOnTuLTz75xOZ1i8UiXnzxRREcHCw0Go0YOXKkSEpKclJrXZfBYBDz5s0TERERwsPDQ7Rt21b885//FEajUTqH97p+duzYUeXv55kzZwohandfs7OzxbRp04SPj4/QarXioYceEnl5eQ1um0KISkseEhEREbkQ1sgQERGRy2KQISIiIpfFIENEREQui0GGiIiIXBaDDBEREbksBhkiIiJyWQwyRERE5LIYZIiIiMhlMcgQkd20bt0ay5Ytq/X5O3fuhEKhuG5zyqaqrveHiG5M5ewGEJHzjBgxAr169bLbl+u+ffvg7e1d6/MHDx6M9PR06HQ6u3w+Ed18GGSIqEZCCJjNZqhUN/51ERgYWKdrq9VqhISE1LdpREQcWiK6Wc2aNQuxsbF49913oVAooFAocOHCBWm4Z/Pmzejbty80Gg12796Ns2fPYuLEiQgODoaPjw/69++Pbdu22Vzzr0MnCoUC//vf/zB58mR4eXmhQ4cO+OGHH6TX/zq0tGrVKvj5+WHLli3o0qULfHx8MHbsWKSnp0vvMZlMeOqpp+Dn54eAgAAsWrQIM2fOxKRJk2r8++7evRtDhw6Fp6cnwsPD8dRTT6GgoMCm7a+88gqmTZsGb29vtGjRAsuXL7e5RkpKCiZOnAgfHx9otVpMnToVmZmZNuf8+OOP6N+/Pzw8PNC8eXNMnjzZ5vXCwkL8/e9/h6+vLyIiIvDJJ5/U2G4iqhmDDNFN6t1330VUVBQeffRRpKenIz09HeHh4dLrzz//PJYsWYKTJ08iMjIS+fn5GD9+PLZv346DBw9i7NixmDBhAlJSUmr8nJdffhlTp07FkSNHMH78eEyfPh05OTnVnl9YWIi33noLX375JXbt2oWUlBQ888wz0uv/93//h9WrV2PlypXYs2cPDAYDNm7cWGMbzp49i7Fjx2LKlCk4cuQIvvnmG+zevRtz5861Oe/NN99Ez549cfDgQTz//POYN28etm7dCgCwWCyYOHEicnJyEBsbi61bt+LcuXO49957pff//PPPmDx5MsaPH4+DBw9i+/btGDBggM1nvP322+jXrx8OHjyIJ554Ao8//jiSkpJqbD8R1aDB+2cTkcsaPny4mDdvns2xHTt2CABi48aNN3x/t27dxPvvvy89b9WqlVi6dKn0HIB44YUXpOf5+fkCgNi8ebPNZ127dk0IIcTKlSsFAJGcnCy9Z/ny5SI4OFh6HhwcLN58803puclkEhEREWLixInVtvPhhx8Ws2fPtjn2xx9/CKVSKYqKiqS2jx071uace++9V4wbN04IIcRvv/0m3NzcREpKivT68ePHBQCxd+9eIYQQUVFRYvr06dW2o1WrVmLGjBnSc4vFIoKCgsSKFSuqfQ8R1Yw9MkRUpX79+tk8z8/PxzPPPIMuXbrAz88PPj4+OHny5A17ZCIjI6U/e3t7Q6vVIisrq9rzvby80K5dO+l5aGiodL5er0dmZqZNL4ebmxv69u1bYxsOHz6MVatWwcfHR3qMGTMGFosF58+fl86LioqyeV9UVBROnjwJADh58iTCw8Nteq26du0KPz8/6ZxDhw5h5MiRNbal8v1QKBQICQmp8X4QUc1Y7EtEVfrr7KNnnnkGW7duxVtvvYX27dvD09MT99xzD0pKSmq8jru7u81zhUIBi8VSp/OFEHVsva38/Hw89thjeOqpp657LSIiokHXrszT0/OG59T1fhBRzdgjQ3QTU6vVMJvNtTp3z549mDVrFiZPnowePXogJCQEFy5ckLeBf6HT6RAcHIx9+/ZJx8xmMw4cOFDj+/r06YMTJ06gffv21z3UarV0Xnx8vM374uPj0aVLFwBAly5dkJqaitTUVOn1EydOIDc3F127dgVQ1tuyffv2Bv89iaj22CNDdBNr3bo1EhIScOHCBfj4+MDf37/aczt06ID169djwoQJUCgUePHFF53Sk/Dkk08iJiYG7du3R+fOnfH+++/j2rVrUCgU1b5n0aJFGDRoEObOnYtHHnkE3t7eOHHiBLZu3YoPPvhAOm/Pnj144403MGnSJGzduhXr1q3Dzz//DAAYNWoUevTogenTp2PZsmUwmUx44oknMHz4cGkY7l//+hdGjhyJdu3a4b777oPJZMIvv/yCRYsWyXtTiG5i7JEhuok988wzcHNzQ9euXREYGFhjvcs777yDZs2aYfDgwZgwYQLGjBmDPn36OLC1ZRYtWoRp06bhwQcfRFRUlFTv4uHhUe17IiMjERsbi9OnT2Po0KHo3bs3XnrpJYSFhdmc9/TTT2P//v3o3bs3Xn31VbzzzjsYM2YMgLIhoE2bNqFZs2YYNmwYRo0ahbZt2+Kbb76R3j9ixAisW7cOP/zwA3r16oXbbrsNe/fuledGEBEAQCEaOvhMROREFosFXbp0wdSpU/HKK6/U+zqtW7fG/PnzMX/+fPs1johkx6ElInIpFy9exG+//Ybhw4fDaDTigw8+wPnz53H//fc7u2lE5AQcWiIil6JUKrFq1Sr0798fQ4YMwdGjR7Ft2zapKJeIbi4cWiIiIiKXxR4ZIiIiclkMMkREROSyGGSIiIjIZTHIEBERkctikCEiIiKXxSBDRERELotBhoiIiFwWgwwRERG5rP8HII+udK+6FqIAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 为对比学习负采样准备词频率分布\n",
    "vocab_size = len(dataset.token2id)\n",
    "embed_size = 128\n",
    "distribution = dataset.get_word_distribution()\n",
    "print(distribution)\n",
    "model = SkipGramNCE(vocab_size, embed_size, distribution)\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.optim import SGD, Adam\n",
    "\n",
    "# 定义静态方法collate_batch批量处理数据，转化为PyTorch可以需要的张量类型\n",
    "class DataCollator:\n",
    "    @classmethod\n",
    "    def collate_batch(cls, batch):\n",
    "        batch = np.array(batch)\n",
    "        input_ids = torch.tensor(batch[:, 0], dtype=torch.long)\n",
    "        labels = torch.tensor(batch[:, 1], dtype=torch.long)\n",
    "        return {'input_ids': input_ids, 'labels': labels}\n",
    "\n",
    "# 定义训练参数以及训练循环\n",
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 1e-3\n",
    "epoch_loss = []\n",
    "\n",
    "data_collator = DataCollator()\n",
    "dataloader = DataLoader(data, batch_size=batch_size, shuffle=True,\\\n",
    "    collate_fn=data_collator.collate_batch)\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)\n",
    "model.zero_grad()\n",
    "model.train()\n",
    "\n",
    "# 需要提前安装tqdm\n",
    "from tqdm import trange\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 训练过程，每步读取数据，送入模型计算损失，并使用PyTorch进行优化\n",
    "with trange(epochs, desc='epoch', ncols=60) as pbar:\n",
    "    for epoch in pbar:\n",
    "        for step, batch in enumerate(dataloader):\n",
    "            loss = model(**batch)\n",
    "            pbar.set_description(f'epoch-{epoch}, loss={loss.item():.4f}')\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            model.zero_grad()\n",
    "        epoch_loss.append(loss.item())\n",
    "    \n",
    "epoch_loss = np.array(epoch_loss)\n",
    "plt.plot(range(len(epoch_loss)), epoch_loss)\n",
    "plt.xlabel('training epoch')\n",
    "plt.ylabel('loss')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9430e9a",
   "metadata": {},
   "source": [
    "TF-IDF加权\n",
    "\n",
    "定义词频率（term frequency）。注意到不同长度的文章词频率会有较大差距，不利于比较和运算，因此可以对词频率取对数。\n",
    "\n",
    "$$\\text{tf}_{t,d} = \\log (\\text{count}(t,d) + 1)$$\n",
    "\n",
    "其中$\\text{count}(t,d)$表示词$t$在文档$d$中出现的次数，为了避免对0取对数，把所有的计数加1。\n",
    "\n",
    "那么如何区分高频词与低频词呢？TF-IDF引入了另一个重要的评价指标——文档频率（document frequency），即一个词在语料库所包含的多少篇文档中出现。在所有文档里出现的词往往是虚词或是常见实词，而只在少量文档里出现的词往往是具有明确含义的实词并且具有很强的文档区分度。用$\\text{df}_t$来表示在多少篇文档中出现了词$t$。\n",
    "\n",
    "为了压低高频词和提升低频词的影响，TF-IDF使用文档频率的倒数，也就是逆向文档频率（inverse document frequency）来对词频率进行加权。这很好理解，一个词的文档频率越高，其倒数就越小，权重就越小。\n",
    "\n",
    "$$\\text{idf}_t = \\log \\frac{N}{\\text{df}_t}$$\n",
    "\n",
    "其中$N$表示文档总数。为了避免分母为0，通常会将分母改为$\\text{df}_t+1$。\n",
    "\n",
    "基于词频率和逆向文档频率，得到TF-IDF的最终值为：\n",
    "\n",
    "$$w_{t,d} = \\text{tf}_{t,d} \\times \\text{idf}_{t}$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f765e353",
   "metadata": {},
   "source": [
    "很多情况下会额外对文档的TF-IDF向量使用L2归一化，使得不同文档的TF-IDF向量具有相同的模长，便于相互比较。\n",
    "下面给出了TF-IDF的代码实现。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9ce8e610",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TFIDF:\n",
    "    def __init__(self, vocab_size, norm='l2', smooth_idf=True,\\\n",
    "                 sublinear_tf=True):\n",
    "        self.vocab_size = vocab_size\n",
    "        self.norm = norm\n",
    "        self.smooth_idf = smooth_idf\n",
    "        self.sublinear_tf = sublinear_tf\n",
    "    \n",
    "    def fit(self, X):\n",
    "        doc_freq = np.zeros(self.vocab_size, dtype=np.float64)\n",
    "        for data in X:\n",
    "            for token_id in set(data):\n",
    "                doc_freq[token_id] += 1\n",
    "        doc_freq += int(self.smooth_idf)\n",
    "        n_samples = len(X) + int(self.smooth_idf)\n",
    "        self.idf = np.log(n_samples / doc_freq) + 1\n",
    "    \n",
    "    def transform(self, X):\n",
    "        assert hasattr(self, 'idf')\n",
    "        term_freq = np.zeros((len(X), self.vocab_size), dtype=np.float64)\n",
    "        for i, data in enumerate(X):\n",
    "            for token in data:\n",
    "                term_freq[i, token] += 1\n",
    "        if self.sublinear_tf:\n",
    "            term_freq = np.log(term_freq + 1)\n",
    "        Y = term_freq * self.idf\n",
    "        if self.norm:\n",
    "            row_norm = (Y**2).sum(axis=1)\n",
    "            row_norm[row_norm == 0] = 1\n",
    "            Y /= np.sqrt(row_norm)[:, None]\n",
    "        return Y\n",
    "    \n",
    "    def fit_transform(self, X):\n",
    "        self.fit(X)\n",
    "        return self.transform(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f32e3a97",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "650b7a27",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77c97957",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
